{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# setup pf client and execution path\n",
    "\n",
    "from promptflow import PFClient\n",
    "import json\n",
    "import os\n",
    "\n",
    "pf = PFClient()\n",
    "\n",
    "root = os.path.join(os.getcwd(), \"../\")\n",
    "flow = os.path.join(root, \"maths-to-code\")\n",
    "data = os.path.join(flow, \"math_data.jsonl\")\n",
    "eval_flow = os.path.join(root, \"../evaluation/eval-accuracy-maths-to-code\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# start batch run of maths-to-code\n",
    "base_run = pf.run(\n",
    "    flow = flow, \n",
    "    data = data, \n",
    "    column_mapping={\"math_question\": \"${data.question}\"},\n",
    "    display_name=\"maths_to_code_batch_run\",\n",
    "    stream=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>inputs.math_question</th>\n",
       "      <th>inputs.line_number</th>\n",
       "      <th>outputs.answer</th>\n",
       "      <th>outputs.code</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>What is the sum of 5 and 3?</td>\n",
       "      <td>0</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>print(5 + 3)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Subtract 7 from 10.</td>\n",
       "      <td>1</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>print(10 - 7)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Multiply 6 by 4.</td>\n",
       "      <td>2</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>print(6 * 4)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Divide 20 by 5.</td>\n",
       "      <td>3</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>print(20 / 5)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>What is the square of 7?</td>\n",
       "      <td>4</td>\n",
       "      <td>49.000000</td>\n",
       "      <td>print(7**2)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>What is the square root of 81?</td>\n",
       "      <td>5</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>import math\\nprint(math.sqrt(81))</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>If a rectangle has a length of 10 and width of...</td>\n",
       "      <td>6</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>length = 10\\nwidth = 5\\narea = length * width\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>A circle has a radius of 7, what is the area? ...</td>\n",
       "      <td>7</td>\n",
       "      <td>153.860000</td>\n",
       "      <td>area = 3.14 * (7**2)\\nprint(area)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Solve for x in the equation 2x + 3 = 9.</td>\n",
       "      <td>8</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>print((9-3)/2)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>What is the value of x if 5x = 25?</td>\n",
       "      <td>9</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>print(25/5)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>A car travels 200 miles in 4 hours. What is th...</td>\n",
       "      <td>10</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>print(200/4)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>A car travels at a speed of 60 mph. How long w...</td>\n",
       "      <td>11</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>print(180 / 60)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>If a car travels at a speed of 40 mph for 2 ho...</td>\n",
       "      <td>12</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>print(40 * 2)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>A rectangle has length = 10 cm and width = 5 c...</td>\n",
       "      <td>13</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>print(10 * 5)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>A circle has radius = 7 cm. What is its circum...</td>\n",
       "      <td>14</td>\n",
       "      <td>43.960000</td>\n",
       "      <td>radius = 7\\ncircumference = 2 * 3.14 * radius\\...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>A triangle has base =10 cm and height =5 cm. W...</td>\n",
       "      <td>15</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>print((10*5)/2)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>What is the slope of the line that passes thro...</td>\n",
       "      <td>16</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>x1, y1 = 2, 3\\nx2, y2 = 4, 7\\nslope = (y2 - y1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>The distance between A and B is 2000km, A is m...</td>\n",
       "      <td>17</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>print(2000/(80+120))</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>The lengths of the two perpendicular sides of ...</td>\n",
       "      <td>18</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>import math\\n\\na = 6\\nb = 8\\nc = math.sqrt(a**...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>A is running with average speed 10km/hour, A a...</td>\n",
       "      <td>19</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>A_distance = 10 * 0.5\\nB_speed = 15\\nB_time = ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 inputs.math_question  inputs.line_number  \\\n",
       "0                         What is the sum of 5 and 3?                   0   \n",
       "1                                 Subtract 7 from 10.                   1   \n",
       "2                                    Multiply 6 by 4.                   2   \n",
       "3                                     Divide 20 by 5.                   3   \n",
       "4                            What is the square of 7?                   4   \n",
       "5                      What is the square root of 81?                   5   \n",
       "6   If a rectangle has a length of 10 and width of...                   6   \n",
       "7   A circle has a radius of 7, what is the area? ...                   7   \n",
       "8             Solve for x in the equation 2x + 3 = 9.                   8   \n",
       "9                  What is the value of x if 5x = 25?                   9   \n",
       "10  A car travels 200 miles in 4 hours. What is th...                  10   \n",
       "11  A car travels at a speed of 60 mph. How long w...                  11   \n",
       "12  If a car travels at a speed of 40 mph for 2 ho...                  12   \n",
       "13  A rectangle has length = 10 cm and width = 5 c...                  13   \n",
       "14  A circle has radius = 7 cm. What is its circum...                  14   \n",
       "15  A triangle has base =10 cm and height =5 cm. W...                  15   \n",
       "16  What is the slope of the line that passes thro...                  16   \n",
       "17  The distance between A and B is 2000km, A is m...                  17   \n",
       "18  The lengths of the two perpendicular sides of ...                  18   \n",
       "19  A is running with average speed 10km/hour, A a...                  19   \n",
       "\n",
       "    outputs.answer                                       outputs.code  \n",
       "0         8.000000                                       print(5 + 3)  \n",
       "1         3.000000                                      print(10 - 7)  \n",
       "2        24.000000                                       print(6 * 4)  \n",
       "3         4.000000                                      print(20 / 5)  \n",
       "4        49.000000                                        print(7**2)  \n",
       "5         9.000000                  import math\\nprint(math.sqrt(81))  \n",
       "6        50.000000  length = 10\\nwidth = 5\\narea = length * width\\...  \n",
       "7       153.860000                  area = 3.14 * (7**2)\\nprint(area)  \n",
       "8         3.000000                                     print((9-3)/2)  \n",
       "9         5.000000                                        print(25/5)  \n",
       "10       50.000000                                       print(200/4)  \n",
       "11        3.000000                                    print(180 / 60)  \n",
       "12       80.000000                                      print(40 * 2)  \n",
       "13       50.000000                                      print(10 * 5)  \n",
       "14       43.960000  radius = 7\\ncircumference = 2 * 3.14 * radius\\...  \n",
       "15       25.000000                                    print((10*5)/2)  \n",
       "16        2.000000  x1, y1 = 2, 3\\nx2, y2 = 4, 7\\nslope = (y2 - y1...  \n",
       "17       10.000000                               print(2000/(80+120))  \n",
       "18       10.000000  import math\\n\\na = 6\\nb = 8\\nc = math.sqrt(a**...  \n",
       "19        0.333333  A_distance = 10 * 0.5\\nB_speed = 15\\nB_time = ...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Show output of flow run\n",
    "pf.get_details(base_run)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# evaluate against the batch run and groundtruth data\n",
    "eval_run = pf.run(\n",
    "    flow = eval_flow, \n",
    "    data = data, \n",
    "    run = base_run,\n",
    "    column_mapping={\"groundtruth\": \"${data.answer}\", \"prediction\": \"${run.outputs.answer}\"},\n",
    "    display_name=\"maths_to_code_eval_run\",\n",
    "    stream=True\n",
    ")\n",
    "\n",
    "pf.get_details(eval_run)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.95, 'error_rate': 0.0}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get metrics of the evaluation flow run\n",
    "pf.get_metrics(eval_run)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualize the flow run and evaluation run with HTML\n",
    "pf.visualize([base_run, eval_run])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run on Azure\n",
    "If you want to run and evaluate your flow on Azure, you can using following example to setup your Azure ML workspace "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azure.identity import DefaultAzureCredential, InteractiveBrowserCredential\n",
    "\n",
    "# init credential\n",
    "try:\n",
    "    credential = DefaultAzureCredential()\n",
    "    # Check if given credential can get token successfully.\n",
    "    credential.get_token(\"https://management.azure.com/.default\")\n",
    "except Exception as ex:\n",
    "    # Fall back to InteractiveBrowserCredential in case DefaultAzureCredential not work\n",
    "    credential = InteractiveBrowserCredential()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from promptflow.azure import PFClient\n",
    "\n",
    "try:\n",
    "    pf = PFClient.from_config(credential=credential)\n",
    "except Exception as ex:\n",
    "    # NOTE: Update following workspace information if not correctly configure before\n",
    "    client_config = {\n",
    "        \"subscription_id\": \"<SUBSCRIPTION_ID>\",\n",
    "        \"resource_group\": \"<RESOURCE_GROUP>\",\n",
    "        \"workspace_name\": \"<AML_WORKSPACE_NAME>\",\n",
    "    }\n",
    "\n",
    "    if client_config[\"subscription_id\"].startswith(\"<\"):\n",
    "        print(\n",
    "            \"please update your <SUBSCRIPTION_ID> <RESOURCE_GROUP> <AML_WORKSPACE_NAME> in notebook cell\"\n",
    "        )\n",
    "        raise ex\n",
    "    else:  # write and reload from config file\n",
    "        import json, os\n",
    "\n",
    "        config_path = \"../.azureml/config.json\"\n",
    "        os.makedirs(os.path.dirname(config_path), exist_ok=True)\n",
    "        with open(config_path, \"w\") as fo:\n",
    "            fo.write(json.dumps(client_config))\n",
    "        pf = PFClient.from_config(credential=credential, path=config_path)\n",
    "\n",
    "print(pf)\n",
    "\n",
    "# NOTE: replace with your own runtime name in your Azure Machine Learning workspace\n",
    "runtime = '<runtime_name>'\n",
    "\n",
    "# NOTE: note that you need to replace <open_ai_connection> and <gpt-35-turbo> with your own connection and deployment name in your Azure Machine Learning workspace\n",
    "connection_mapping = {\"code_gen\": {\"connection\": \"<my_azure_open_ai_connection>\", \"deployment_name\": \"<gpt-35-turbo>\"}}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# batch run of maths to code\n",
    "\n",
    "base_run = pf.run(\n",
    "    flow = flow,\n",
    "    data = data,\n",
    "    column_mapping = {\"math_question\": \"${data.question}\"},\n",
    "    connections = connection_mapping,\n",
    "    runtime = runtime,\n",
    "    stream = True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>inputs.math_question</th>\n",
       "      <th>outputs.code</th>\n",
       "      <th>outputs.answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Subtract 7 from 10.</td>\n",
       "      <td>print(10 - 7)</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>What is the sum of 5 and 3?</td>\n",
       "      <td>print(5+3)</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>What is the square of 7?</td>\n",
       "      <td>print(7**2)</td>\n",
       "      <td>49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A car travels 200 miles in 4 hours. What is th...</td>\n",
       "      <td>print(200 / 4)</td>\n",
       "      <td>50.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Multiply 6 by 4.</td>\n",
       "      <td>print(6 * 4)</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>What is the value of x if 5x = 25?</td>\n",
       "      <td>print(25/5)</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>If a car travels at a speed of 40 mph for 2 ho...</td>\n",
       "      <td>print(40 * 2)</td>\n",
       "      <td>80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>A car travels at a speed of 60 mph. How long w...</td>\n",
       "      <td>print(180 / 60)</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>A triangle has base =10 cm and height =5 cm. W...</td>\n",
       "      <td>print((10*5)/2)</td>\n",
       "      <td>25.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Divide 20 by 5.</td>\n",
       "      <td>print(20 / 5)</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>If a rectangle has a length of 10 and width of...</td>\n",
       "      <td>print(10 * 5)</td>\n",
       "      <td>50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>What is the square root of 81?</td>\n",
       "      <td>import math\\nprint(math.sqrt(81))</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Solve for x in the equation 2x + 3 = 9.</td>\n",
       "      <td>print((9-3)/2)</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>A rectangle has length = 10 cm and width = 5 c...</td>\n",
       "      <td>length = 10\\ncm_width = 5\\narea = length * wid...</td>\n",
       "      <td>name 'width' is not defined</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>A circle has a radius of 7, what is the area? ...</td>\n",
       "      <td>pi = 3.14\\nradius = 7\\narea = pi * radius ** 2...</td>\n",
       "      <td>153.86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>What is the slope of the line that passes thro...</td>\n",
       "      <td>slope = (7-3)/(4-2)\\nprint(slope)</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>The distance between A and B is 2000km, A is m...</td>\n",
       "      <td>distance = 2000\\nspeed_A = 80\\nspeed_B = 120\\n...</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>A circle has radius = 7 cm. What is its circum...</td>\n",
       "      <td>pi = 3.14\\nradius = 7\\ncircumference = 2 * pi ...</td>\n",
       "      <td>43.96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>The lengths of the two perpendicular sides of ...</td>\n",
       "      <td>import math\\n\\na = 6\\nb = 8\\nc = math.sqrt(a**...</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>A is running with average speed 10km/hour, A a...</td>\n",
       "      <td>distance_A = 10 * (0.5)\\n\\nspeed_B = 15\\n\\ntim...</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 inputs.math_question  \\\n",
       "0                                 Subtract 7 from 10.   \n",
       "1                         What is the sum of 5 and 3?   \n",
       "2                            What is the square of 7?   \n",
       "3   A car travels 200 miles in 4 hours. What is th...   \n",
       "4                                    Multiply 6 by 4.   \n",
       "5                  What is the value of x if 5x = 25?   \n",
       "6   If a car travels at a speed of 40 mph for 2 ho...   \n",
       "7   A car travels at a speed of 60 mph. How long w...   \n",
       "8   A triangle has base =10 cm and height =5 cm. W...   \n",
       "9                                     Divide 20 by 5.   \n",
       "10  If a rectangle has a length of 10 and width of...   \n",
       "11                     What is the square root of 81?   \n",
       "12            Solve for x in the equation 2x + 3 = 9.   \n",
       "13  A rectangle has length = 10 cm and width = 5 c...   \n",
       "14  A circle has a radius of 7, what is the area? ...   \n",
       "15  What is the slope of the line that passes thro...   \n",
       "16  The distance between A and B is 2000km, A is m...   \n",
       "17  A circle has radius = 7 cm. What is its circum...   \n",
       "18  The lengths of the two perpendicular sides of ...   \n",
       "19  A is running with average speed 10km/hour, A a...   \n",
       "\n",
       "                                         outputs.code  \\\n",
       "0                                       print(10 - 7)   \n",
       "1                                          print(5+3)   \n",
       "2                                         print(7**2)   \n",
       "3                                      print(200 / 4)   \n",
       "4                                        print(6 * 4)   \n",
       "5                                         print(25/5)   \n",
       "6                                       print(40 * 2)   \n",
       "7                                     print(180 / 60)   \n",
       "8                                     print((10*5)/2)   \n",
       "9                                       print(20 / 5)   \n",
       "10                                      print(10 * 5)   \n",
       "11                  import math\\nprint(math.sqrt(81))   \n",
       "12                                     print((9-3)/2)   \n",
       "13  length = 10\\ncm_width = 5\\narea = length * wid...   \n",
       "14  pi = 3.14\\nradius = 7\\narea = pi * radius ** 2...   \n",
       "15                  slope = (7-3)/(4-2)\\nprint(slope)   \n",
       "16  distance = 2000\\nspeed_A = 80\\nspeed_B = 120\\n...   \n",
       "17  pi = 3.14\\nradius = 7\\ncircumference = 2 * pi ...   \n",
       "18  import math\\n\\na = 6\\nb = 8\\nc = math.sqrt(a**...   \n",
       "19  distance_A = 10 * (0.5)\\n\\nspeed_B = 15\\n\\ntim...   \n",
       "\n",
       "                 outputs.answer  \n",
       "0                             3  \n",
       "1                             8  \n",
       "2                            49  \n",
       "3                          50.0  \n",
       "4                            24  \n",
       "5                           5.0  \n",
       "6                            80  \n",
       "7                           3.0  \n",
       "8                          25.0  \n",
       "9                           4.0  \n",
       "10                           50  \n",
       "11                          9.0  \n",
       "12                          3.0  \n",
       "13  name 'width' is not defined  \n",
       "14                       153.86  \n",
       "15                          2.0  \n",
       "16                         10.0  \n",
       "17                        43.96  \n",
       "18                         10.0  \n",
       "19                               "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get output of flow run\n",
    "pf.get_details(base_run)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# evaluation run against base run\n",
    "\n",
    "eval_run = pf.run(\n",
    "    flow = eval_flow, \n",
    "    data = data, \n",
    "    run = base_run,\n",
    "    column_mapping={\"groundtruth\": \"${data.answer}\", \"prediction\": \"${run.outputs.answer}\"},\n",
    "    runtime = runtime,\n",
    "    stream = True,\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>inputs.groundtruth</th>\n",
       "      <th>inputs.prediction</th>\n",
       "      <th>outputs.score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4.00</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.00</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>24.00</td>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>49.00</td>\n",
       "      <td>49</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.00</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>50.00</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>8.00</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>3.00</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>43.96</td>\n",
       "      <td>43.96</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>3.00</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>153.86</td>\n",
       "      <td>153.86</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>9.00</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>80.00</td>\n",
       "      <td>80</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>25.00</td>\n",
       "      <td>25.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>50.00</td>\n",
       "      <td>50.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>2.00</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>50.00</td>\n",
       "      <td>name 'width' is not defined</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>10.00</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>10.00</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1.00</td>\n",
       "      <td></td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    inputs.groundtruth            inputs.prediction  outputs.score\n",
       "0                 4.00                          4.0              1\n",
       "1                 3.00                            3              1\n",
       "2                24.00                           24              1\n",
       "3                49.00                           49              1\n",
       "4                 5.00                          5.0              1\n",
       "5                50.00                           50              1\n",
       "6                 8.00                            8              1\n",
       "7                 3.00                          3.0              1\n",
       "8                43.96                        43.96              1\n",
       "9                 3.00                          3.0              1\n",
       "10              153.86                       153.86              1\n",
       "11                9.00                          9.0              1\n",
       "12               80.00                           80              1\n",
       "13               25.00                         25.0              1\n",
       "14               50.00                         50.0              1\n",
       "15                2.00                          2.0              1\n",
       "16               50.00  name 'width' is not defined             -1\n",
       "17               10.00                         10.0              1\n",
       "18               10.00                         10.0              1\n",
       "19                1.00                                          -1"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get output of evaluation run\n",
    "pf.get_details(eval_run)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"accuracy\": 0.9,\n",
      "    \"error_rate\": 0.1\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "metrics = pf.get_metrics(eval_run)\n",
    "print(json.dumps(metrics, indent=4))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pf",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
